{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using MLRUN function locally, as a Kubernetes Job, and in a Workflow\n",
    "  --------------------------------------------------------------------\n",
    "\n",
    "#### **notebook how-to's**\n",
    "* Write and test code in a notebook.\n",
    "* Convert it to a containerized image.\n",
    "* Run it on a Kubernetes cluster with shared file or object storage.\n",
    "* Run it in an automated workflow."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='top'></a>\n",
    "#### **steps**\n",
    "**[intall mlrun](#install)**<br>\n",
    "**[define a new function and its dependencies](#define-function)**<br>\n",
    "**[test the function code and pipeline locally](#test-locally)**<br>\n",
    "**[define cluster jobs and build images](#build)**<br>\n",
    "**[deploy (build) the function container](#deploy-build)**<br>\n",
    "**[run the function on the cluster](#run-on-cluster)**<br>\n",
    "**[create and run a KubeFlow Pipeline](#create-pipeline)**<br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id=\"install\" ></a>\n",
    "______________________________________________\n",
    "### **install mlrun**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Uncomment this to install mlrun package, restart the kernel after\n",
    "\n",
    "# !pip install -U mlrun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set the UI external URL (will generate ui hyperlinks)\n",
    "# %env MLRUN_UI_URL=http://<mlrun-ui-url>:<port>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "______________________________________________"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='define-function'></a>\n",
    "### **define a new function and its dependencies**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mlrun: ignore\n",
    "# do not remove the comment above (it is a directive to nuclio, ignore that cell during build)\n",
    "# if the nuclio-jupyter package is not installed run !pip install nuclio-jupyter and restart the kernel\n",
    "import nuclio"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We use `%nuclio` magic commands to set package dependencies and configuration:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "%nuclio: setting spec.build.baseImage to 'mlrun/mlrun:unstable'\n"
     ]
    }
   ],
   "source": [
    "%nuclio cmd -c pip install pandas\n",
    "%nuclio config spec.build.baseImage = \"mlrun/mlrun\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The ```DataItem```s and the ```context``` within which they are logged are described in the following ```mlrun``` modules (they are included here only for type clarity)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mlrun.execution import MLClientCtx\n",
    "from mlrun.datastore import DataItem\n",
    "from mlrun.artifacts import get_model, update_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "def training(context: MLClientCtx, p1: int = 1, p2: int = 2) -> None:\n",
    "    \"\"\"Train a model.\n",
    "\n",
    "    :param context: The runtime context object.\n",
    "    :param p1: A model parameter.\n",
    "    :param p2: Another model parameter.\n",
    "    \"\"\"\n",
    "    # access input metadata, values, and inputs\n",
    "    print(f\"Run: {context.name} (uid={context.uid})\")\n",
    "    print(f\"Params: p1={p1}, p2={p2}\")\n",
    "    context.logger.info(\"started training\")\n",
    "\n",
    "    # <insert training code here>\n",
    "\n",
    "    # log the run results (scalar values)\n",
    "    context.log_result(\"accuracy\", p1 * 2)\n",
    "    context.log_result(\"loss\", p1 * 3)\n",
    "\n",
    "    # add a lable/tag to this run\n",
    "    context.set_label(\"category\", \"tests\")\n",
    "\n",
    "    # log a simple artifact + label the artifact\n",
    "    # If you want to upload a local file to the artifact repo add src_path=<local-path>\n",
    "    context.log_artifact(\"somefile\", body=b\"abc is 123\", local_path=\"myfile.txt\")\n",
    "\n",
    "    # create a dataframe artifact\n",
    "    df = pd.DataFrame([{\"A\": 10, \"B\": 100}, {\"A\": 11, \"B\": 110}, {\"A\": 12, \"B\": 120}])\n",
    "    context.log_dataset(\"mydf\", df=df)\n",
    "\n",
    "    # Log an ML Model artifact, add metrics, params, and labels to it\n",
    "    # and place it in a subdir ('models') under artifacts path\n",
    "    context.log_model(\n",
    "        \"mymodel\",\n",
    "        body=b\"abc is 123\",\n",
    "        model_file=\"model.txt\",\n",
    "        metrics={\"accuracy\": 0.85},\n",
    "        parameters={\"xx\": \"abc\"},\n",
    "        labels={\"framework\": \"xgboost\"},\n",
    "        artifact_path=context.artifact_subpath(\"models\"),\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def validation(context: MLClientCtx, model: DataItem) -> None:\n",
    "    \"\"\"Model validation.\n",
    "\n",
    "    Dummy validation function.\n",
    "\n",
    "    :param context: The runtime context object.\n",
    "    :param model: The extimated model object.\n",
    "    \"\"\"\n",
    "    # access input metadata, values, files, and secrets (passwords)\n",
    "    print(f\"Run: {context.name} (uid={context.uid})\")\n",
    "    context.logger.info(\"started validation\")\n",
    "\n",
    "    # get the model file, class (metadata), and extra_data (dict of key: DataItem)\n",
    "    model_file, model_obj, _ = get_model(model)\n",
    "\n",
    "    # update model object elements and data\n",
    "    update_model(model_obj, parameters={\"one_more\": 5})\n",
    "\n",
    "    print(f\"path to local copy of model file - {model_file}\")\n",
    "    print(\"parameters:\", model_obj.parameters)\n",
    "    print(\"metrics:\", model_obj.metrics)\n",
    "    context.log_artifact(\"validation\", body=b\"<b> validated </b>\", format=\"html\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following end-code annotation tells ```nuclio``` to stop parsing the notebook from this cell. _**Please do not remove this cell**_:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mlrun: end-code"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "______________________________________________"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='test-locally'></a>\n",
    "### **test the function code and pipeline locally**\n",
    "The functions above can be tested locally. Parameters, inputs, and outputs can be specified in the API or the `Task` object.\n",
    "\n",
    "We create a ```function``` which defines the runtime environment (type, code, image, ..) and ```run()``` a job or experiments using that function.\n",
    "\n",
    "We use the ```local``` runtime by default, later on we will use a ```job``` runtime for running containers, and can use other distributed runners like MpiJob, Spark, Dask, and Nuclio.\n",
    "\n",
    "In each run we can specify the function, inputs, parameters/hyper-parameters, etc... For more details, see the [mlrun_basics notebook](mlrun_basics.ipynb)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mlrun import run_local, code_to_function, mlconf, new_task\n",
    "from mlrun.platforms.other import auto_mount\n",
    "\n",
    "mlconf.dbpath = mlconf.dbpath or \"http://mlrun-api:8080\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b> define the artifact location</b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os import path\n",
    "\n",
    "out = mlconf.artifact_path or path.abspath(\"./data\")\n",
    "# {{run.uid}} will be substituted with the run id, so output will be written to different directoried per run\n",
    "artifact_path = path.join(out, \"{{run.uid}}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### _running and linking multiple tasks_\n",
    "In this example we run two functions, ```training``` and ```validation``` and we pass the result from one to the other.\n",
    "We will see in the ```job``` example that linking works even when the tasks are run in a workflow on different processes or containers."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```run_local()``` will run our task on a local function:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Run the training function. Functions can have multiple handlers/methods, here we call the ```training``` handler:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[mlrun] 2020-06-08 20:55:48,958 starting run mlrun-0e0d5d-training uid=ee93118dc24f4b849c5ef525f81144ea  -> http://mlrun-api:8080\n",
      "Run: mlrun-0e0d5d-training (uid=ee93118dc24f4b849c5ef525f81144ea)\n",
      "Params: p1=5, p2=2\n",
      "[mlrun] 2020-06-08 20:55:48,983 started training\n",
      "[mlrun] 2020-06-08 20:55:49,004 log artifact somefile at /User/ml3/mlrun/examples/data/myfile.txt, size: 10, db: Y\n",
      "[mlrun] 2020-06-08 20:55:49,042 log artifact mydf at /User/ml3/mlrun/examples/data/mydf.csv, size: 32, db: Y\n",
      "[mlrun] 2020-06-08 20:55:49,066 log artifact mymodel at /User/ml3/mlrun/examples/data/models/, size: 10, db: Y\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style> \n",
       ".dictlist {\n",
       "  background-color: #b3edff; \n",
       "  text-align: center; \n",
       "  margin: 4px; \n",
       "  border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;}\n",
       ".artifact {\n",
       "  cursor: pointer; \n",
       "  background-color: #ffe6cc; \n",
       "  text-align: left; \n",
       "  margin: 4px; border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;\n",
       "}\n",
       "div.block.hidden {\n",
       "  display: none;\n",
       "}\n",
       ".clickable {\n",
       "  cursor: pointer;\n",
       "}\n",
       ".ellipsis {\n",
       "  display: inline-block;\n",
       "  max-width: 60px;\n",
       "  white-space: nowrap;\n",
       "  overflow: hidden;\n",
       "  text-overflow: ellipsis;\n",
       "}\n",
       ".master-wrapper {\n",
       "  display: flex;\n",
       "  flex-flow: row nowrap;\n",
       "  justify-content: flex-start;\n",
       "  align-items: stretch;\n",
       "}\n",
       ".master-tbl {\n",
       "  flex: 3\n",
       "}\n",
       ".master-wrapper > div {\n",
       "  margin: 4px;\n",
       "  padding: 10px;\n",
       "}\n",
       "iframe.fileview {\n",
       "  border: 0 none;\n",
       "  height: 100%;\n",
       "  width: 100%;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       ".pane-header-title {\n",
       "  width: 80%;\n",
       "  font-weight: 500;\n",
       "}\n",
       ".pane-header {\n",
       "  line-height: 1;\n",
       "  background-color: #ffe6cc;\n",
       "  padding: 3px;\n",
       "}\n",
       ".pane-header .close {\n",
       "  font-size: 20px;\n",
       "  font-weight: 700;\n",
       "  float: right;\n",
       "  margin-top: -5px;\n",
       "}\n",
       ".master-wrapper .right-pane {\n",
       "  border: 1px inset silver;\n",
       "  width: 40%;\n",
       "  min-height: 300px;\n",
       "  flex: 3\n",
       "  min-width: 500px;\n",
       "}\n",
       ".master-wrapper * {\n",
       "  box-sizing: border-box;\n",
       "}\n",
       "</style><script>\n",
       "function copyToClipboard(fld) {\n",
       "    if (document.queryCommandSupported && document.queryCommandSupported('copy')) {\n",
       "        var textarea = document.createElement('textarea');\n",
       "        textarea.textContent = fld.innerHTML;\n",
       "        textarea.style.position = 'fixed';\n",
       "        document.body.appendChild(textarea);\n",
       "        textarea.select();\n",
       "\n",
       "        try {\n",
       "            return document.execCommand('copy'); // Security exception may be thrown by some browsers.\n",
       "        } catch (ex) {\n",
       "\n",
       "        } finally {\n",
       "            document.body.removeChild(textarea);\n",
       "        }\n",
       "    }\n",
       "}\n",
       "function expandPanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName');\n",
       "  console.log(el.title);\n",
       "\n",
       "  document.querySelector(panelName + \"-title\").innerHTML = el.title\n",
       "  iframe = document.querySelector(panelName + \"-body\");\n",
       "  \n",
       "  const tblcss = `<style> body { font-family: Arial, Helvetica, sans-serif;}\n",
       "    #csv { margin-bottom: 15px; }\n",
       "    #csv table { border-collapse: collapse;}\n",
       "    #csv table td { padding: 4px 8px; border: 1px solid silver;} </style>`;\n",
       "\n",
       "  function csvToHtmlTable(str) {\n",
       "    return '<div id=\"csv\"><table><tr><td>' +  str.replace(/[\\n\\r]+$/g, '').replace(/[\\n\\r]+/g, '</td></tr><tr><td>')\n",
       "      .replace(/,/g, '</td><td>') + '</td></tr></table></div>';\n",
       "  }\n",
       "  \n",
       "  function reqListener () {\n",
       "    if (el.title.endsWith(\".csv\")) {\n",
       "      iframe.setAttribute(\"srcdoc\", tblcss + csvToHtmlTable(this.responseText));\n",
       "    } else {\n",
       "      iframe.setAttribute(\"srcdoc\", this.responseText);\n",
       "    }  \n",
       "    console.log(this.responseText);\n",
       "  }\n",
       "\n",
       "  const oReq = new XMLHttpRequest();\n",
       "  oReq.addEventListener(\"load\", reqListener);\n",
       "  oReq.open(\"GET\", el.title);\n",
       "  oReq.send();\n",
       "  \n",
       "  \n",
       "  //iframe.src = el.title;\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.remove(\"hidden\");\n",
       "  }\n",
       "}\n",
       "function closePanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName')\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (!resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.add(\"hidden\");\n",
       "  }\n",
       "}\n",
       "\n",
       "</script>\n",
       "<div class=\"master-wrapper\">\n",
       "  <div class=\"block master-tbl\"><div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>project</th>\n",
       "      <th>uid</th>\n",
       "      <th>iter</th>\n",
       "      <th>start</th>\n",
       "      <th>state</th>\n",
       "      <th>name</th>\n",
       "      <th>labels</th>\n",
       "      <th>inputs</th>\n",
       "      <th>parameters</th>\n",
       "      <th>results</th>\n",
       "      <th>artifacts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>default</td>\n",
       "      <td><div title=\"ee93118dc24f4b849c5ef525f81144ea\"><a href=\"https://mlrun-ui.default-tenant.app.yh55.iguazio-cd2.com/projects/default/jobs/ee93118dc24f4b849c5ef525f81144ea/info\" target=\"_blank\" >...f81144ea</a></div></td>\n",
       "      <td>0</td>\n",
       "      <td>Jun 08 20:55:48</td>\n",
       "      <td>completed</td>\n",
       "      <td>mlrun-0e0d5d-training</td>\n",
       "      <td><div class=\"dictlist\">v3io_user=admin</div><div class=\"dictlist\">kind=handler</div><div class=\"dictlist\">owner=admin</div><div class=\"dictlist\">host=jupyter-65887d7ffb-5jsn2</div><div class=\"dictlist\">category=tests</div></td>\n",
       "      <td></td>\n",
       "      <td><div class=\"dictlist\">p1=5</div></td>\n",
       "      <td><div class=\"dictlist\">accuracy=10</div><div class=\"dictlist\">loss=15</div></td>\n",
       "      <td><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"resulta509908b\" title=\"/files/ml3/mlrun/examples/data/myfile.txt\">somefile</div><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"resulta509908b\" title=\"/files/ml3/mlrun/examples/data/mydf.csv\">mydf</div><div title=\"/User/ml3/mlrun/examples/data/models\">mymodel</div></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div></div>\n",
       "  <div id=\"resulta509908b-pane\" class=\"right-pane block hidden\">\n",
       "    <div class=\"pane-header\">\n",
       "      <span id=\"resulta509908b-title\" class=\"pane-header-title\">Title</span>\n",
       "      <span onclick=\"closePanel(this)\" paneName=\"resulta509908b\" class=\"close clickable\">&times;</span>\n",
       "    </div>\n",
       "    <iframe class=\"fileview\" id=\"resulta509908b-body\"></iframe>\n",
       "  </div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "to track results use .show() or .logs() or in CLI: \n",
      "!mlrun get run ee93118dc24f4b849c5ef525f81144ea --project default , !mlrun logs ee93118dc24f4b849c5ef525f81144ea --project default\n",
      "[mlrun] 2020-06-08 20:55:49,101 run executed, status=completed\n"
     ]
    }
   ],
   "source": [
    "train_run = run_local(new_task(handler=training, params={\"p1\": 5}, artifact_path=out))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After the function runs it generates the result widget, you can click the `model` artifact to see its content."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 10,\n",
       " 'loss': 15,\n",
       " 'somefile': '/User/ml3/mlrun/examples/data/myfile.txt',\n",
       " 'mydf': 'store://default/mlrun-0e0d5d-training_mydf#ee93118dc24f4b849c5ef525f81144ea',\n",
       " 'mymodel': 'store://default/mlrun-0e0d5d-training_mymodel#ee93118dc24f4b849c5ef525f81144ea'}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_run.outputs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The output from the first training function is passed to the validation function, let's run it:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[mlrun] 2020-06-08 20:59:57,156 starting run mlrun-22e231-validation uid=2bc7df39f243453a8ab92bb87483fe03  -> http://mlrun-api:8080\n",
      "Run: mlrun-22e231-validation (uid=2bc7df39f243453a8ab92bb87483fe03)\n",
      "[mlrun] 2020-06-08 20:59:57,214 started validation\n",
      "path to local copy of model file - /User/ml3/mlrun/examples/data/models/model.txt\n",
      "parameters: {'xx': 'abc'}\n",
      "metrics: {'accuracy': 0.85}\n",
      "[mlrun] 2020-06-08 20:59:57,252 log artifact validation at /User/ml3/mlrun/examples/data/validation.html, size: 18, db: Y\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style> \n",
       ".dictlist {\n",
       "  background-color: #b3edff; \n",
       "  text-align: center; \n",
       "  margin: 4px; \n",
       "  border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;}\n",
       ".artifact {\n",
       "  cursor: pointer; \n",
       "  background-color: #ffe6cc; \n",
       "  text-align: left; \n",
       "  margin: 4px; border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;\n",
       "}\n",
       "div.block.hidden {\n",
       "  display: none;\n",
       "}\n",
       ".clickable {\n",
       "  cursor: pointer;\n",
       "}\n",
       ".ellipsis {\n",
       "  display: inline-block;\n",
       "  max-width: 60px;\n",
       "  white-space: nowrap;\n",
       "  overflow: hidden;\n",
       "  text-overflow: ellipsis;\n",
       "}\n",
       ".master-wrapper {\n",
       "  display: flex;\n",
       "  flex-flow: row nowrap;\n",
       "  justify-content: flex-start;\n",
       "  align-items: stretch;\n",
       "}\n",
       ".master-tbl {\n",
       "  flex: 3\n",
       "}\n",
       ".master-wrapper > div {\n",
       "  margin: 4px;\n",
       "  padding: 10px;\n",
       "}\n",
       "iframe.fileview {\n",
       "  border: 0 none;\n",
       "  height: 100%;\n",
       "  width: 100%;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       ".pane-header-title {\n",
       "  width: 80%;\n",
       "  font-weight: 500;\n",
       "}\n",
       ".pane-header {\n",
       "  line-height: 1;\n",
       "  background-color: #ffe6cc;\n",
       "  padding: 3px;\n",
       "}\n",
       ".pane-header .close {\n",
       "  font-size: 20px;\n",
       "  font-weight: 700;\n",
       "  float: right;\n",
       "  margin-top: -5px;\n",
       "}\n",
       ".master-wrapper .right-pane {\n",
       "  border: 1px inset silver;\n",
       "  width: 40%;\n",
       "  min-height: 300px;\n",
       "  flex: 3\n",
       "  min-width: 500px;\n",
       "}\n",
       ".master-wrapper * {\n",
       "  box-sizing: border-box;\n",
       "}\n",
       "</style><script>\n",
       "function copyToClipboard(fld) {\n",
       "    if (document.queryCommandSupported && document.queryCommandSupported('copy')) {\n",
       "        var textarea = document.createElement('textarea');\n",
       "        textarea.textContent = fld.innerHTML;\n",
       "        textarea.style.position = 'fixed';\n",
       "        document.body.appendChild(textarea);\n",
       "        textarea.select();\n",
       "\n",
       "        try {\n",
       "            return document.execCommand('copy'); // Security exception may be thrown by some browsers.\n",
       "        } catch (ex) {\n",
       "\n",
       "        } finally {\n",
       "            document.body.removeChild(textarea);\n",
       "        }\n",
       "    }\n",
       "}\n",
       "function expandPanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName');\n",
       "  console.log(el.title);\n",
       "\n",
       "  document.querySelector(panelName + \"-title\").innerHTML = el.title\n",
       "  iframe = document.querySelector(panelName + \"-body\");\n",
       "  \n",
       "  const tblcss = `<style> body { font-family: Arial, Helvetica, sans-serif;}\n",
       "    #csv { margin-bottom: 15px; }\n",
       "    #csv table { border-collapse: collapse;}\n",
       "    #csv table td { padding: 4px 8px; border: 1px solid silver;} </style>`;\n",
       "\n",
       "  function csvToHtmlTable(str) {\n",
       "    return '<div id=\"csv\"><table><tr><td>' +  str.replace(/[\\n\\r]+$/g, '').replace(/[\\n\\r]+/g, '</td></tr><tr><td>')\n",
       "      .replace(/,/g, '</td><td>') + '</td></tr></table></div>';\n",
       "  }\n",
       "  \n",
       "  function reqListener () {\n",
       "    if (el.title.endsWith(\".csv\")) {\n",
       "      iframe.setAttribute(\"srcdoc\", tblcss + csvToHtmlTable(this.responseText));\n",
       "    } else {\n",
       "      iframe.setAttribute(\"srcdoc\", this.responseText);\n",
       "    }  \n",
       "    console.log(this.responseText);\n",
       "  }\n",
       "\n",
       "  const oReq = new XMLHttpRequest();\n",
       "  oReq.addEventListener(\"load\", reqListener);\n",
       "  oReq.open(\"GET\", el.title);\n",
       "  oReq.send();\n",
       "  \n",
       "  \n",
       "  //iframe.src = el.title;\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.remove(\"hidden\");\n",
       "  }\n",
       "}\n",
       "function closePanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName')\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (!resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.add(\"hidden\");\n",
       "  }\n",
       "}\n",
       "\n",
       "</script>\n",
       "<div class=\"master-wrapper\">\n",
       "  <div class=\"block master-tbl\"><div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>project</th>\n",
       "      <th>uid</th>\n",
       "      <th>iter</th>\n",
       "      <th>start</th>\n",
       "      <th>state</th>\n",
       "      <th>name</th>\n",
       "      <th>labels</th>\n",
       "      <th>inputs</th>\n",
       "      <th>parameters</th>\n",
       "      <th>results</th>\n",
       "      <th>artifacts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>default</td>\n",
       "      <td><div title=\"2bc7df39f243453a8ab92bb87483fe03\"><a href=\"https://mlrun-ui.default-tenant.app.yh55.iguazio-cd2.com/projects/default/jobs/2bc7df39f243453a8ab92bb87483fe03/info\" target=\"_blank\" >...7483fe03</a></div></td>\n",
       "      <td>0</td>\n",
       "      <td>Jun 08 20:59:57</td>\n",
       "      <td>completed</td>\n",
       "      <td>mlrun-22e231-validation</td>\n",
       "      <td><div class=\"dictlist\">v3io_user=admin</div><div class=\"dictlist\">kind=handler</div><div class=\"dictlist\">owner=admin</div><div class=\"dictlist\">host=jupyter-65887d7ffb-5jsn2</div></td>\n",
       "      <td><div title=\"store://default/mlrun-0e0d5d-training_mymodel#ee93118dc24f4b849c5ef525f81144ea\">model</div></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"result9c285c4c\" title=\"/files/ml3/mlrun/examples/data/validation.html\">validation</div></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div></div>\n",
       "  <div id=\"result9c285c4c-pane\" class=\"right-pane block hidden\">\n",
       "    <div class=\"pane-header\">\n",
       "      <span id=\"result9c285c4c-title\" class=\"pane-header-title\">Title</span>\n",
       "      <span onclick=\"closePanel(this)\" paneName=\"result9c285c4c\" class=\"close clickable\">&times;</span>\n",
       "    </div>\n",
       "    <iframe class=\"fileview\" id=\"result9c285c4c-body\"></iframe>\n",
       "  </div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "to track results use .show() or .logs() or in CLI: \n",
      "!mlrun get run 2bc7df39f243453a8ab92bb87483fe03 --project default , !mlrun logs 2bc7df39f243453a8ab92bb87483fe03 --project default\n",
      "[mlrun] 2020-06-08 20:59:57,285 run executed, status=completed\n"
     ]
    }
   ],
   "source": [
    "model = train_run.outputs[\"mymodel\"]\n",
    "\n",
    "validation_run = run_local(\n",
    "    new_task(handler=validation, inputs={\"model\": model}, artifact_path=out)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "______________________________________________"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id=\"build\"></a>\n",
    "### **define cluster jobs and build images**\n",
    "\n",
    "In order to use our function in a cluster we need to package our code and dependencies.\n",
    "\n",
    "The ```code_to_function``` call will automatically generate a ```function``` object from the current notebook (or a specified file) with its list of dependencies and runtime configuration."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create an ML function from the notebook, attache it to iguazio data fabric (v3io)\n",
    "trainer = code_to_function(name=\"my-trainer\", kind=\"job\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The functions need shared storage (file or object) media to pass and store artifacts.\n",
    "\n",
    "You can add _**Kubernetes**_ resources like volumes, environment variables, secrets, cpu/mem/gpu, etc. to a function.\n",
    "\n",
    "```mlrun``` uses _**KubeFlow**_ modifiers (apply) to configure resources, you can build your own or use predefined ones e.g. for [AWS resources](https://github.com/kubeflow/pipelines/blob/master/sdk/python/kfp/aws.py).\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### _**Option 1: Using file volumes for artifacts**_\n",
    "If your are using [Iguazio data science platform](https://www.iguazio.com/) use the `mount_v3io()` auto-mount modifier.<br>\n",
    "if you use other k8s PVC volumes you can use the `mlrun.platforms.mount_pvc(..)` modifier with the required params.\n",
    "\n",
    "We will use the `auto_mount()` modifier which auto selects between the options, you can set the PVC volume config via env var:\n",
    "```\n",
    "    MLRUN_PVC_MOUNT=<pvc-name>:<mount-path>\n",
    "```\n",
    "\n",
    "Applying ```mount_v3io()``` will attach the function to Iguazio's real-time data fabric (mounted by default to _**home**_ of the current user).\n",
    "\n",
    "**Note**: if the notebook is not on the managed platform (running remotely) you need to create and use a v3io secret, run:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`kubectl create -n <namespace> secret generic my-v3io --from-literal=accessKey=<your access key> --from-literal=username=<your user name> --type v3io/fuse`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "and use: `trainer.apply(mount_v3io(user='admin', secret='my-v3io'))`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "So for our current ```training``` function, when using Iguazio data science platform run:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<mlrun.runtimes.kubejob.KubejobRuntime at 0x7f3548051f98>"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# for auto choice between Iguazio platform and k8s PVC\n",
    "# should set the env var for PVC: MLRUN_PVC_MOUNT=<pvc-name>:<mount-path>, or use mount_pvc()\n",
    "trainer.apply(auto_mount())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Uncomment for use with shared PVC volumes, e.g. using the NFS Share in the local k8s install\n",
    "# from mlrun.platforms import mount_pvc\n",
    "# trainer.apply(mount_pvc('nfsvol', 'nfsvol', '/home/jovyan/data'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### _**Option 2: Using AWS S3 for artifacts**_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In AWS you can use S3 and need to have a `secret` with AWS credentials. An AWS secret can be created with the following command line:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`kubectl create -n <namespace> secret generic my-aws --from-literal=AWS_ACCESS_KEY_ID=<access key> --from-literal=AWS_SECRET_ACCESS_KEY=<secret key>`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To use the secret:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from kfp.aws import use_aws_secret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# trainer.apply(use_aws_secret(secret_name='my-aws'))\n",
    "# out = 's3://<your-bucket-name>/jobs/{{run.uid}}'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "______________________________________________"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id=\"deploy-build\"></a>\n",
    "### **deploy (build) the function container**\n",
    "\n",
    "The `deploy()` command will build a custom container image (create a cluster build job) from the outlined function dependencies.\n",
    "\n",
    "If a pre-built container image already exists, pass the `image` name instead. _**Note that the code and params can be updated per run without building a new image**_.\n",
    "\n",
    "The image is stored in a container repository, and by default it uses the repository configured on the MLRun API service, you can specify your own docker registry by first creating a secret, and adding that secret name to the build configuration:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": "`kubectl create -n <namespace> secret docker-registry my-docker --docker-server=https://index.docker.io/v1/ --docker-username=<your-user> --docker-password=<your-password> --docker-email=<your-email>`"
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "and run this: `trainer.build_config(image='target/image:tag', secret='my_docker')`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer.deploy(with_mlrun=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id=\"run-on-cluster\"></a>\n",
    "### **run the function on the cluster**\n",
    "\n",
    "\n",
    "In case we made changes to the code, ```with_code``` will inject the latest code into the function (it doesn't require a new build)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<mlrun.runtimes.kubejob.KubejobRuntime at 0x7f3548051f98>"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.with_code()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create the base task (common to both steps), and set the output path and experiment label\n",
    "base_task = new_task(artifact_path=out).set_label(\"stage\", \"dev\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[mlrun] 2020-06-08 21:19:38,337 starting run my-training uid=772f68e2195241709e299eb4d2892545  -> http://mlrun-api:8080\n",
      "[mlrun] 2020-06-08 21:19:38,473 Job is running in the background, pod: my-training-wjhfk\n",
      "[mlrun] 2020-06-08 21:19:54,532 starting local run: main.py # training\n",
      "Run: my-training (uid=772f68e2195241709e299eb4d2892545)\n",
      "Params: p1=9, p2=2\n",
      "[mlrun] 2020-06-08 21:19:54,553 started training\n",
      "[mlrun] 2020-06-08 21:19:54,576 log artifact somefile at /User/ml3/mlrun/examples/data/myfile.txt, size: 10, db: Y\n",
      "[mlrun] 2020-06-08 21:19:54,604 log artifact mydf at /User/ml3/mlrun/examples/data/mydf.csv, size: 32, db: Y\n",
      "[mlrun] 2020-06-08 21:19:54,622 log artifact mymodel at /User/ml3/mlrun/examples/data/models/, size: 10, db: Y\n",
      "\n",
      "[mlrun] 2020-06-08 21:19:54,637 run executed, status=completed\n",
      "final state: succeeded\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style> \n",
       ".dictlist {\n",
       "  background-color: #b3edff; \n",
       "  text-align: center; \n",
       "  margin: 4px; \n",
       "  border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;}\n",
       ".artifact {\n",
       "  cursor: pointer; \n",
       "  background-color: #ffe6cc; \n",
       "  text-align: left; \n",
       "  margin: 4px; border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;\n",
       "}\n",
       "div.block.hidden {\n",
       "  display: none;\n",
       "}\n",
       ".clickable {\n",
       "  cursor: pointer;\n",
       "}\n",
       ".ellipsis {\n",
       "  display: inline-block;\n",
       "  max-width: 60px;\n",
       "  white-space: nowrap;\n",
       "  overflow: hidden;\n",
       "  text-overflow: ellipsis;\n",
       "}\n",
       ".master-wrapper {\n",
       "  display: flex;\n",
       "  flex-flow: row nowrap;\n",
       "  justify-content: flex-start;\n",
       "  align-items: stretch;\n",
       "}\n",
       ".master-tbl {\n",
       "  flex: 3\n",
       "}\n",
       ".master-wrapper > div {\n",
       "  margin: 4px;\n",
       "  padding: 10px;\n",
       "}\n",
       "iframe.fileview {\n",
       "  border: 0 none;\n",
       "  height: 100%;\n",
       "  width: 100%;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       ".pane-header-title {\n",
       "  width: 80%;\n",
       "  font-weight: 500;\n",
       "}\n",
       ".pane-header {\n",
       "  line-height: 1;\n",
       "  background-color: #ffe6cc;\n",
       "  padding: 3px;\n",
       "}\n",
       ".pane-header .close {\n",
       "  font-size: 20px;\n",
       "  font-weight: 700;\n",
       "  float: right;\n",
       "  margin-top: -5px;\n",
       "}\n",
       ".master-wrapper .right-pane {\n",
       "  border: 1px inset silver;\n",
       "  width: 40%;\n",
       "  min-height: 300px;\n",
       "  flex: 3\n",
       "  min-width: 500px;\n",
       "}\n",
       ".master-wrapper * {\n",
       "  box-sizing: border-box;\n",
       "}\n",
       "</style><script>\n",
       "function copyToClipboard(fld) {\n",
       "    if (document.queryCommandSupported && document.queryCommandSupported('copy')) {\n",
       "        var textarea = document.createElement('textarea');\n",
       "        textarea.textContent = fld.innerHTML;\n",
       "        textarea.style.position = 'fixed';\n",
       "        document.body.appendChild(textarea);\n",
       "        textarea.select();\n",
       "\n",
       "        try {\n",
       "            return document.execCommand('copy'); // Security exception may be thrown by some browsers.\n",
       "        } catch (ex) {\n",
       "\n",
       "        } finally {\n",
       "            document.body.removeChild(textarea);\n",
       "        }\n",
       "    }\n",
       "}\n",
       "function expandPanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName');\n",
       "  console.log(el.title);\n",
       "\n",
       "  document.querySelector(panelName + \"-title\").innerHTML = el.title\n",
       "  iframe = document.querySelector(panelName + \"-body\");\n",
       "  \n",
       "  const tblcss = `<style> body { font-family: Arial, Helvetica, sans-serif;}\n",
       "    #csv { margin-bottom: 15px; }\n",
       "    #csv table { border-collapse: collapse;}\n",
       "    #csv table td { padding: 4px 8px; border: 1px solid silver;} </style>`;\n",
       "\n",
       "  function csvToHtmlTable(str) {\n",
       "    return '<div id=\"csv\"><table><tr><td>' +  str.replace(/[\\n\\r]+$/g, '').replace(/[\\n\\r]+/g, '</td></tr><tr><td>')\n",
       "      .replace(/,/g, '</td><td>') + '</td></tr></table></div>';\n",
       "  }\n",
       "  \n",
       "  function reqListener () {\n",
       "    if (el.title.endsWith(\".csv\")) {\n",
       "      iframe.setAttribute(\"srcdoc\", tblcss + csvToHtmlTable(this.responseText));\n",
       "    } else {\n",
       "      iframe.setAttribute(\"srcdoc\", this.responseText);\n",
       "    }  \n",
       "    console.log(this.responseText);\n",
       "  }\n",
       "\n",
       "  const oReq = new XMLHttpRequest();\n",
       "  oReq.addEventListener(\"load\", reqListener);\n",
       "  oReq.open(\"GET\", el.title);\n",
       "  oReq.send();\n",
       "  \n",
       "  \n",
       "  //iframe.src = el.title;\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.remove(\"hidden\");\n",
       "  }\n",
       "}\n",
       "function closePanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName')\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (!resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.add(\"hidden\");\n",
       "  }\n",
       "}\n",
       "\n",
       "</script>\n",
       "<div class=\"master-wrapper\">\n",
       "  <div class=\"block master-tbl\"><div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>project</th>\n",
       "      <th>uid</th>\n",
       "      <th>iter</th>\n",
       "      <th>start</th>\n",
       "      <th>state</th>\n",
       "      <th>name</th>\n",
       "      <th>labels</th>\n",
       "      <th>inputs</th>\n",
       "      <th>parameters</th>\n",
       "      <th>results</th>\n",
       "      <th>artifacts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>default</td>\n",
       "      <td><div title=\"772f68e2195241709e299eb4d2892545\"><a href=\"https://mlrun-ui.default-tenant.app.yh55.iguazio-cd2.com/projects/default/jobs/772f68e2195241709e299eb4d2892545/info\" target=\"_blank\" >...d2892545</a></div></td>\n",
       "      <td>0</td>\n",
       "      <td>Jun 08 21:19:54</td>\n",
       "      <td>completed</td>\n",
       "      <td>my-training</td>\n",
       "      <td><div class=\"dictlist\">stage=dev</div><div class=\"dictlist\">v3io_user=admin</div><div class=\"dictlist\">kind=job</div><div class=\"dictlist\">owner=admin</div><div class=\"dictlist\">host=my-training-wjhfk</div><div class=\"dictlist\">category=tests</div></td>\n",
       "      <td></td>\n",
       "      <td><div class=\"dictlist\">p1=9</div></td>\n",
       "      <td><div class=\"dictlist\">accuracy=18</div><div class=\"dictlist\">loss=27</div></td>\n",
       "      <td><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"result10feea3b\" title=\"/files/ml3/mlrun/examples/data/myfile.txt\">somefile</div><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"result10feea3b\" title=\"/files/ml3/mlrun/examples/data/mydf.csv\">mydf</div><div title=\"/User/ml3/mlrun/examples/data/models\">mymodel</div></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div></div>\n",
       "  <div id=\"result10feea3b-pane\" class=\"right-pane block hidden\">\n",
       "    <div class=\"pane-header\">\n",
       "      <span id=\"result10feea3b-title\" class=\"pane-header-title\">Title</span>\n",
       "      <span onclick=\"closePanel(this)\" paneName=\"result10feea3b\" class=\"close clickable\">&times;</span>\n",
       "    </div>\n",
       "    <iframe class=\"fileview\" id=\"result10feea3b-body\"></iframe>\n",
       "  </div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "to track results use .show() or .logs() or in CLI: \n",
      "!mlrun get run 772f68e2195241709e299eb4d2892545 --project default , !mlrun logs 772f68e2195241709e299eb4d2892545 --project default\n",
      "[mlrun] 2020-06-08 21:19:57,693 run executed, status=completed\n"
     ]
    }
   ],
   "source": [
    "# run our training task, with hyper params, and select the one with max accuracy\n",
    "train_task = new_task(\n",
    "    name=\"my-training\", handler=\"training\", params={\"p1\": 9}, base=base_task\n",
    ")\n",
    "train_run = trainer.run(train_task)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[mlrun] 2020-06-08 21:20:06,468 starting run my-trainer-validation uid=d78eb87823404b9fb6c5147b87b6e537  -> http://mlrun-api:8080\n",
      "[mlrun] 2020-06-08 21:20:06,575 Job is running in the background, pod: my-trainer-validation-5gjwk\n",
      "[mlrun] 2020-06-08 21:20:10,674 starting local run: main.py # validation\n",
      "Run: my-trainer-validation (uid=d78eb87823404b9fb6c5147b87b6e537)\n",
      "[mlrun] 2020-06-08 21:20:10,697 started validation\n",
      "path to local copy of model file - /User/ml3/mlrun/examples/data/models/model.txt\n",
      "parameters: {'xx': 'abc'}\n",
      "metrics: {'accuracy': 0.85}\n",
      "[mlrun] 2020-06-08 21:20:10,727 log artifact validation at /User/ml3/mlrun/examples/data/validation.html, size: 18, db: Y\n",
      "\n",
      "[mlrun] 2020-06-08 21:20:10,737 run executed, status=completed\n",
      "final state: succeeded\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style> \n",
       ".dictlist {\n",
       "  background-color: #b3edff; \n",
       "  text-align: center; \n",
       "  margin: 4px; \n",
       "  border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;}\n",
       ".artifact {\n",
       "  cursor: pointer; \n",
       "  background-color: #ffe6cc; \n",
       "  text-align: left; \n",
       "  margin: 4px; border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;\n",
       "}\n",
       "div.block.hidden {\n",
       "  display: none;\n",
       "}\n",
       ".clickable {\n",
       "  cursor: pointer;\n",
       "}\n",
       ".ellipsis {\n",
       "  display: inline-block;\n",
       "  max-width: 60px;\n",
       "  white-space: nowrap;\n",
       "  overflow: hidden;\n",
       "  text-overflow: ellipsis;\n",
       "}\n",
       ".master-wrapper {\n",
       "  display: flex;\n",
       "  flex-flow: row nowrap;\n",
       "  justify-content: flex-start;\n",
       "  align-items: stretch;\n",
       "}\n",
       ".master-tbl {\n",
       "  flex: 3\n",
       "}\n",
       ".master-wrapper > div {\n",
       "  margin: 4px;\n",
       "  padding: 10px;\n",
       "}\n",
       "iframe.fileview {\n",
       "  border: 0 none;\n",
       "  height: 100%;\n",
       "  width: 100%;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       ".pane-header-title {\n",
       "  width: 80%;\n",
       "  font-weight: 500;\n",
       "}\n",
       ".pane-header {\n",
       "  line-height: 1;\n",
       "  background-color: #ffe6cc;\n",
       "  padding: 3px;\n",
       "}\n",
       ".pane-header .close {\n",
       "  font-size: 20px;\n",
       "  font-weight: 700;\n",
       "  float: right;\n",
       "  margin-top: -5px;\n",
       "}\n",
       ".master-wrapper .right-pane {\n",
       "  border: 1px inset silver;\n",
       "  width: 40%;\n",
       "  min-height: 300px;\n",
       "  flex: 3\n",
       "  min-width: 500px;\n",
       "}\n",
       ".master-wrapper * {\n",
       "  box-sizing: border-box;\n",
       "}\n",
       "</style><script>\n",
       "function copyToClipboard(fld) {\n",
       "    if (document.queryCommandSupported && document.queryCommandSupported('copy')) {\n",
       "        var textarea = document.createElement('textarea');\n",
       "        textarea.textContent = fld.innerHTML;\n",
       "        textarea.style.position = 'fixed';\n",
       "        document.body.appendChild(textarea);\n",
       "        textarea.select();\n",
       "\n",
       "        try {\n",
       "            return document.execCommand('copy'); // Security exception may be thrown by some browsers.\n",
       "        } catch (ex) {\n",
       "\n",
       "        } finally {\n",
       "            document.body.removeChild(textarea);\n",
       "        }\n",
       "    }\n",
       "}\n",
       "function expandPanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName');\n",
       "  console.log(el.title);\n",
       "\n",
       "  document.querySelector(panelName + \"-title\").innerHTML = el.title\n",
       "  iframe = document.querySelector(panelName + \"-body\");\n",
       "  \n",
       "  const tblcss = `<style> body { font-family: Arial, Helvetica, sans-serif;}\n",
       "    #csv { margin-bottom: 15px; }\n",
       "    #csv table { border-collapse: collapse;}\n",
       "    #csv table td { padding: 4px 8px; border: 1px solid silver;} </style>`;\n",
       "\n",
       "  function csvToHtmlTable(str) {\n",
       "    return '<div id=\"csv\"><table><tr><td>' +  str.replace(/[\\n\\r]+$/g, '').replace(/[\\n\\r]+/g, '</td></tr><tr><td>')\n",
       "      .replace(/,/g, '</td><td>') + '</td></tr></table></div>';\n",
       "  }\n",
       "  \n",
       "  function reqListener () {\n",
       "    if (el.title.endsWith(\".csv\")) {\n",
       "      iframe.setAttribute(\"srcdoc\", tblcss + csvToHtmlTable(this.responseText));\n",
       "    } else {\n",
       "      iframe.setAttribute(\"srcdoc\", this.responseText);\n",
       "    }  \n",
       "    console.log(this.responseText);\n",
       "  }\n",
       "\n",
       "  const oReq = new XMLHttpRequest();\n",
       "  oReq.addEventListener(\"load\", reqListener);\n",
       "  oReq.open(\"GET\", el.title);\n",
       "  oReq.send();\n",
       "  \n",
       "  \n",
       "  //iframe.src = el.title;\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.remove(\"hidden\");\n",
       "  }\n",
       "}\n",
       "function closePanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName')\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (!resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.add(\"hidden\");\n",
       "  }\n",
       "}\n",
       "\n",
       "</script>\n",
       "<div class=\"master-wrapper\">\n",
       "  <div class=\"block master-tbl\"><div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>project</th>\n",
       "      <th>uid</th>\n",
       "      <th>iter</th>\n",
       "      <th>start</th>\n",
       "      <th>state</th>\n",
       "      <th>name</th>\n",
       "      <th>labels</th>\n",
       "      <th>inputs</th>\n",
       "      <th>parameters</th>\n",
       "      <th>results</th>\n",
       "      <th>artifacts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>default</td>\n",
       "      <td><div title=\"d78eb87823404b9fb6c5147b87b6e537\"><a href=\"https://mlrun-ui.default-tenant.app.yh55.iguazio-cd2.com/projects/default/jobs/d78eb87823404b9fb6c5147b87b6e537/info\" target=\"_blank\" >...87b6e537</a></div></td>\n",
       "      <td>0</td>\n",
       "      <td>Jun 08 21:20:10</td>\n",
       "      <td>completed</td>\n",
       "      <td>my-trainer-validation</td>\n",
       "      <td><div class=\"dictlist\">stage=dev</div><div class=\"dictlist\">v3io_user=admin</div><div class=\"dictlist\">kind=job</div><div class=\"dictlist\">owner=admin</div><div class=\"dictlist\">host=my-trainer-validation-5gjwk</div></td>\n",
       "      <td><div title=\"store://default/my-training_mymodel#772f68e2195241709e299eb4d2892545\">model</div></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"resultf9a81259\" title=\"/files/ml3/mlrun/examples/data/validation.html\">validation</div></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div></div>\n",
       "  <div id=\"resultf9a81259-pane\" class=\"right-pane block hidden\">\n",
       "    <div class=\"pane-header\">\n",
       "      <span id=\"resultf9a81259-title\" class=\"pane-header-title\">Title</span>\n",
       "      <span onclick=\"closePanel(this)\" paneName=\"resultf9a81259\" class=\"close clickable\">&times;</span>\n",
       "    </div>\n",
       "    <iframe class=\"fileview\" id=\"resultf9a81259-body\"></iframe>\n",
       "  </div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "to track results use .show() or .logs() or in CLI: \n",
      "!mlrun get run d78eb87823404b9fb6c5147b87b6e537 --project default , !mlrun logs d78eb87823404b9fb6c5147b87b6e537 --project default\n",
      "[mlrun] 2020-06-08 21:20:12,707 run executed, status=completed\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<mlrun.model.RunObject at 0x7f3548058240>"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# running validation, use the model result from the previous step\n",
    "model = train_run.outputs[\"mymodel\"]\n",
    "trainer.run(base_task, handler=\"validation\", inputs={\"model\": model}, watch=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "______________________________________________"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id=\"create-pipeline\"></a>\n",
    "### **create and run a KubeFlow pipeline**\n",
    "\n",
    "KubeFlow pipelines are used for workflow automation--we compose a graph of functions and specify parameters, inputs and outputs.\n",
    "\n",
    "As illustrated below, we can chain the outputs and inputs of the pipeline steps."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "import kfp\n",
    "from kfp import dsl\n",
    "from mlrun import run_pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "@dsl.pipeline(name=\"job test\", description=\"demonstrating mlrun usage\")\n",
    "def job_pipeline(p1: int = 9) -> None:\n",
    "    \"\"\"Define our pipeline.\n",
    "\n",
    "    :param p1: A model parameter.\n",
    "    \"\"\"\n",
    "\n",
    "    train = trainer.as_step(handler=\"training\", params={\"p1\": p1}, outputs=[\"mymodel\"])\n",
    "\n",
    "    validate = trainer.as_step(\n",
    "        handler=\"validation\",\n",
    "        inputs={\"model\": train.outputs[\"mymodel\"]},\n",
    "        outputs=[\"validation\"],\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The job pipeline can compiled to a yaml file that can be used for debugging:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "kfp.compiler.Compiler().compile(job_pipeline, \"jobpipe.yaml\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### running the pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pipeline results are stored at the `artifact_path` location:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "However, by adding ```/{{workflow.uid}}``` to the path ```mlrun``` will generate a unique folder per workflow."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "artifact_path = \"v3io:///users/admin/kfp/{{workflow.uid}}/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Experiment link <a href=\"https://dashboard.default-tenant.app.yh55.iguazio-cd2.com/pipelines/#/experiments/details/5ed3fba2-162e-4e42-86ed-e0e037e40b20\" target=\"_blank\" >here</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run link <a href=\"https://dashboard.default-tenant.app.yh55.iguazio-cd2.com/pipelines/#/runs/details/a9243134-039a-4e38-9de3-dbcb90b54864\" target=\"_blank\" >here</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[mlrun] 2020-06-08 21:21:01,934 Pipeline run id=a9243134-039a-4e38-9de3-dbcb90b54864, check UI or DB for progress\n"
     ]
    }
   ],
   "source": [
    "arguments = {\"p1\": 8}\n",
    "run_id = run_pipeline(\n",
    "    job_pipeline, arguments, experiment=\"my-job\", artifact_path=artifact_path\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[top](#top)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style> \n",
       ".dictlist {\n",
       "  background-color: #b3edff; \n",
       "  text-align: center; \n",
       "  margin: 4px; \n",
       "  border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;}\n",
       ".artifact {\n",
       "  cursor: pointer; \n",
       "  background-color: #ffe6cc; \n",
       "  text-align: left; \n",
       "  margin: 4px; border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;\n",
       "}\n",
       "div.block.hidden {\n",
       "  display: none;\n",
       "}\n",
       ".clickable {\n",
       "  cursor: pointer;\n",
       "}\n",
       ".ellipsis {\n",
       "  display: inline-block;\n",
       "  max-width: 60px;\n",
       "  white-space: nowrap;\n",
       "  overflow: hidden;\n",
       "  text-overflow: ellipsis;\n",
       "}\n",
       ".master-wrapper {\n",
       "  display: flex;\n",
       "  flex-flow: row nowrap;\n",
       "  justify-content: flex-start;\n",
       "  align-items: stretch;\n",
       "}\n",
       ".master-tbl {\n",
       "  flex: 3\n",
       "}\n",
       ".master-wrapper > div {\n",
       "  margin: 4px;\n",
       "  padding: 10px;\n",
       "}\n",
       "iframe.fileview {\n",
       "  border: 0 none;\n",
       "  height: 100%;\n",
       "  width: 100%;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       ".pane-header-title {\n",
       "  width: 80%;\n",
       "  font-weight: 500;\n",
       "}\n",
       ".pane-header {\n",
       "  line-height: 1;\n",
       "  background-color: #ffe6cc;\n",
       "  padding: 3px;\n",
       "}\n",
       ".pane-header .close {\n",
       "  font-size: 20px;\n",
       "  font-weight: 700;\n",
       "  float: right;\n",
       "  margin-top: -5px;\n",
       "}\n",
       ".master-wrapper .right-pane {\n",
       "  border: 1px inset silver;\n",
       "  width: 40%;\n",
       "  min-height: 300px;\n",
       "  flex: 3\n",
       "  min-width: 500px;\n",
       "}\n",
       ".master-wrapper * {\n",
       "  box-sizing: border-box;\n",
       "}\n",
       "</style><script>\n",
       "function copyToClipboard(fld) {\n",
       "    if (document.queryCommandSupported && document.queryCommandSupported('copy')) {\n",
       "        var textarea = document.createElement('textarea');\n",
       "        textarea.textContent = fld.innerHTML;\n",
       "        textarea.style.position = 'fixed';\n",
       "        document.body.appendChild(textarea);\n",
       "        textarea.select();\n",
       "\n",
       "        try {\n",
       "            return document.execCommand('copy'); // Security exception may be thrown by some browsers.\n",
       "        } catch (ex) {\n",
       "\n",
       "        } finally {\n",
       "            document.body.removeChild(textarea);\n",
       "        }\n",
       "    }\n",
       "}\n",
       "function expandPanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName');\n",
       "  console.log(el.title);\n",
       "\n",
       "  document.querySelector(panelName + \"-title\").innerHTML = el.title\n",
       "  iframe = document.querySelector(panelName + \"-body\");\n",
       "  \n",
       "  const tblcss = `<style> body { font-family: Arial, Helvetica, sans-serif;}\n",
       "    #csv { margin-bottom: 15px; }\n",
       "    #csv table { border-collapse: collapse;}\n",
       "    #csv table td { padding: 4px 8px; border: 1px solid silver;} </style>`;\n",
       "\n",
       "  function csvToHtmlTable(str) {\n",
       "    return '<div id=\"csv\"><table><tr><td>' +  str.replace(/[\\n\\r]+$/g, '').replace(/[\\n\\r]+/g, '</td></tr><tr><td>')\n",
       "      .replace(/,/g, '</td><td>') + '</td></tr></table></div>';\n",
       "  }\n",
       "  \n",
       "  function reqListener () {\n",
       "    if (el.title.endsWith(\".csv\")) {\n",
       "      iframe.setAttribute(\"srcdoc\", tblcss + csvToHtmlTable(this.responseText));\n",
       "    } else {\n",
       "      iframe.setAttribute(\"srcdoc\", this.responseText);\n",
       "    }  \n",
       "    console.log(this.responseText);\n",
       "  }\n",
       "\n",
       "  const oReq = new XMLHttpRequest();\n",
       "  oReq.addEventListener(\"load\", reqListener);\n",
       "  oReq.open(\"GET\", el.title);\n",
       "  oReq.send();\n",
       "  \n",
       "  \n",
       "  //iframe.src = el.title;\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.remove(\"hidden\");\n",
       "  }\n",
       "}\n",
       "function closePanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName')\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (!resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.add(\"hidden\");\n",
       "  }\n",
       "}\n",
       "\n",
       "</script>\n",
       "<div class=\"master-wrapper\">\n",
       "  <div class=\"block master-tbl\"><div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>project</th>\n",
       "      <th>uid</th>\n",
       "      <th>iter</th>\n",
       "      <th>start</th>\n",
       "      <th>state</th>\n",
       "      <th>name</th>\n",
       "      <th>labels</th>\n",
       "      <th>inputs</th>\n",
       "      <th>parameters</th>\n",
       "      <th>results</th>\n",
       "      <th>artifacts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>default</td>\n",
       "      <td><div title=\"3f8012b6ff874236a474e62606335fc8\"><a href=\"https://mlrun-ui.default-tenant.app.yh55.iguazio-cd2.com/projects/default/jobs/3f8012b6ff874236a474e62606335fc8/info\" target=\"_blank\" >...06335fc8</a></div></td>\n",
       "      <td>0</td>\n",
       "      <td>Jun 08 21:21:19</td>\n",
       "      <td>completed</td>\n",
       "      <td>my-trainer-validation</td>\n",
       "      <td><div class=\"dictlist\">v3io_user=admin</div><div class=\"dictlist\">owner=admin</div><div class=\"dictlist\">workflow=a9243134-039a-4e38-9de3-dbcb90b54864</div><div class=\"dictlist\">kind=job</div><div class=\"dictlist\">host=my-trainer-validation-gtbdf</div></td>\n",
       "      <td><div title=\"store://default/my-trainer-training_mymodel#a9243134-039a-4e38-9de3-dbcb90b54864\">model</div></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"resultea348e54\" title=\"files/v3io/users/admin/kfp/a9243134-039a-4e38-9de3-dbcb90b54864/validation.html\">validation</div></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>default</td>\n",
       "      <td><div title=\"145b71871abf43ee956501580121511c\"><a href=\"https://mlrun-ui.default-tenant.app.yh55.iguazio-cd2.com/projects/default/jobs/145b71871abf43ee956501580121511c/info\" target=\"_blank\" >...0121511c</a></div></td>\n",
       "      <td>0</td>\n",
       "      <td>Jun 08 21:21:09</td>\n",
       "      <td>completed</td>\n",
       "      <td>my-trainer-training</td>\n",
       "      <td><div class=\"dictlist\">v3io_user=admin</div><div class=\"dictlist\">owner=admin</div><div class=\"dictlist\">workflow=a9243134-039a-4e38-9de3-dbcb90b54864</div><div class=\"dictlist\">kind=job</div><div class=\"dictlist\">host=my-trainer-training-fr5fj</div><div class=\"dictlist\">category=tests</div></td>\n",
       "      <td></td>\n",
       "      <td><div class=\"dictlist\">p1=8</div></td>\n",
       "      <td><div class=\"dictlist\">accuracy=16</div><div class=\"dictlist\">loss=24</div></td>\n",
       "      <td><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"resultea348e54\" title=\"files/v3io/users/admin/kfp/a9243134-039a-4e38-9de3-dbcb90b54864/myfile.txt\">somefile</div><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"resultea348e54\" title=\"files/v3io/users/admin/kfp/a9243134-039a-4e38-9de3-dbcb90b54864/mydf.csv\">mydf</div><div title=\"v3io:///users/admin/kfp/a9243134-039a-4e38-9de3-dbcb90b54864/models/\">mymodel</div></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div></div>\n",
       "  <div id=\"resultea348e54-pane\" class=\"right-pane block hidden\">\n",
       "    <div class=\"pane-header\">\n",
       "      <span id=\"resultea348e54-title\" class=\"pane-header-title\">Title</span>\n",
       "      <span onclick=\"closePanel(this)\" paneName=\"resultea348e54\" class=\"close clickable\">&times;</span>\n",
       "    </div>\n",
       "    <iframe class=\"fileview\" id=\"resultea348e54-body\"></iframe>\n",
       "  </div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from mlrun import wait_for_pipeline_completion, get_run_db\n",
    "\n",
    "wait_for_pipeline_completion(run_id)\n",
    "db = get_run_db().connect()\n",
    "db.list_runs(project=\"default\", labels=f\"workflow={run_id}\").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
